{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-17 18:21:55.980159: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import youtokentome as yttm\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('train-syllable.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "train_X = [d[0] for d in data]\n",
    "train_Y = [d[1] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test-syllable.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "test_X = [d[0] for d in data]\n",
    "test_Y = [d[1] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_X + train_Y + test_X + test_Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training parameters\n",
      "  input: text.txt\n",
      "  model: syllable.model\n",
      "  vocab_size: 300\n",
      "  n_threads: 8\n",
      "  character_coverage: 1\n",
      "  pad: 0\n",
      "  unk: 2\n",
      "  bos: 3\n",
      "  eos: 1\n",
      "\n",
      "reading file...\n",
      "learning bpe...\n",
      "number of unique characters in the training data: 28\n",
      "number of deleted characters: 0\n",
      "number of unique characters left: 28\n",
      "model saved to: syllable.model\n"
     ]
    }
   ],
   "source": [
    "_ = yttm.BPE.train(data='text.txt', vocab_size=300, model='syllable.model',\n",
    "              pad_id=0, unk_id=2, bos_id=3, eos_id=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "bpe = yttm.BPE(model='syllable.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpe.vocab_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD = 0\n",
    "EOS = 1\n",
    "UNK = 2\n",
    "GO = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 from_dict_size, to_dict_size, learning_rate, beam_width = 10):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_out, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(encoder_embedding, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(to_dict_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            decoder_cells = attention(encoder_out, self.X_seq_len)\n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embedding, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "            \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "            predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = predicting_helper,\n",
    "                    initial_state = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state),\n",
    "                    output_layer = dense)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = predicting_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.greedy = predicting_decoder_output.sample_id\n",
    "            self.greedy = tf.identity(self.greedy,name='greedy')\n",
    "        \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            \n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(encoder_out, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            decoder_cell = attention(encoder_out_tiled, X_seq_len_tiled, reuse=True)\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cell,\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = decoder_cell.zero_state(batch_size * beam_width, tf.float32).clone(\n",
    "                    cell_state = encoder_state_tiled),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense,\n",
    "                length_penalty_weight = 0.0)\n",
    "            \n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = tf.reduce_max(self.X_seq_len))\n",
    "            \n",
    "            self.beam = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "            self.beam = tf.identity(self.beam,name='beam')\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 5e-4\n",
    "batch_size = 32\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = bpe.vocab_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/2295251586.py:1: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/2295251586.py:2: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1858933321.py:17: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1858933321.py:23: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1858933321.py:33: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1858933321.py:35: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-17 18:21:58.373496: I tensorflow/core/platform/profile_utils/cpu_utils.cc:109] CPU Frequency: 2112000000 Hz\n",
      "2022-09-17 18:21:58.373933: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3535f30 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2022-09-17 18:21:58.373948: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2022-09-17 18:21:58.375027: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2022-09-17 18:21:58.377049: E tensorflow/stream_executor/cuda/cuda_driver.cc:282] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
      "2022-09-17 18:21:58.377070: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: husein-MS-7D31\n",
      "2022-09-17 18:21:58.377074: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: husein-MS-7D31\n",
      "2022-09-17 18:21:58.377123: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 470.141.3\n",
      "2022-09-17 18:21:58.377137: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 470.141.3\n",
      "2022-09-17 18:21:58.377141: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:310] kernel version seems to match DSO: 470.141.3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1858933321.py:99: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/2295251586.py:4: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, embedded_size, vocab_size, vocab_size, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/3508881337.py:1: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/3508881337.py:1: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'lstm/model.ckpt'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['adpis',\n",
       "  'terbongkang',\n",
       "  'memijit',\n",
       "  'ordi',\n",
       "  'dinaskan',\n",
       "  'plag',\n",
       "  'pendayung',\n",
       "  'sambung-bersambung',\n",
       "  'prausul',\n",
       "  'betari'],\n",
       " ['ad.pis',\n",
       "  'ter.bong.kang',\n",
       "  'me.mi.jit',\n",
       "  'or.di',\n",
       "  'di.nas.kan',\n",
       "  'plag',\n",
       "  'pen.da.yung',\n",
       "  'sam.bung.ber.sam.bung',\n",
       "  'pra.u.sul',\n",
       "  'be.ta.ri'])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X[:10], train_Y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(strings):\n",
    "    encoded = bpe.encode(strings, output_type=yttm.OutputType.ID)\n",
    "    for i in range(len(encoded)):\n",
    "        encoded[i].append(1)\n",
    "        \n",
    "    return pad_sentence_batch(encoded, 0)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x = encode(train_X[: 5])\n",
    "batch_y = encode(train_Y[: 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5.6956706, None]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.cost, model.optimizer], feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: encode(test_X[:1])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|█| 4635/4635 [01:45<00:00, 43.81it/s, accuracy=1, cost=0.00\n",
      "minibatch loop: 100%|█| 244/244 [00:01<00:00, 230.19it/s, accuracy=1, cost=0.001"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 0.018719, training avg acc 0.995121\n",
      "epoch 1, testing avg loss 0.079549, testing avg acc 0.988486\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "import numpy as np\n",
    "\n",
    "batch_size = 8\n",
    "\n",
    "for e in range(1):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = encode(train_X[i: index])\n",
    "        batch_y = encode(train_Y[i: index])\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "        \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = encode(test_X[i: index])\n",
    "        batch_y = encode(test_Y[i: index])\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lstm/model.ckpt'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['me.mi.jit<EOS>']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "berjalankan = bpe.encode('memijit', output_type=yttm.OutputType.ID) + [1]\n",
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [berjalankan]})\n",
    "bpe.decode(greedy.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hu.sein<EOS>']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "berjalankan = bpe.encode('husein', output_type=yttm.OutputType.ID) + [1]\n",
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [berjalankan]})\n",
    "bpe.decode(greedy.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a.yam.a.yam<EOS>']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "berjalankan = bpe.encode('ayam-ayam', output_type=yttm.OutputType.ID) + [1]\n",
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [berjalankan]})\n",
    "bpe.decode(greedy.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ke.se.da.pan<EOS>']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "berjalankan = bpe.encode('kesedapan', output_type=yttm.OutputType.ID) + [1]\n",
    "greedy, beam = sess.run([model.greedy, model.beam], feed_dict = {model.X: [berjalankan]})\n",
    "bpe.decode(greedy.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((False, 'bi.Oa.dat'), (True, 'som.pek'))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def replace_same_length(l, r):\n",
    "    l = l.replace('-', '')\n",
    "    if len(l) != len(r.replace('.', '')):\n",
    "        return False, r\n",
    "    \n",
    "    index = {}\n",
    "    i = 0\n",
    "    for no, c in enumerate(r):\n",
    "        if c not in '.O':\n",
    "            index[no] = i\n",
    "            i += 1\n",
    "    index = {v: k for k, v in index.items()}\n",
    "    \n",
    "    r = list(r)\n",
    "    \n",
    "    for no, c in enumerate(l):\n",
    "        if c != r[index[no]]:\n",
    "            r[index[no]] = c\n",
    "            \n",
    "    return True, ''.join(r)\n",
    "\n",
    "replace_same_length('biadat', 'bi.Oa.dat'), replace_same_length('sompek', 'som.pe/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(True, 'a.yam.a.yam')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "replace_same_length('ayam-ayam', 'a.yau.a.yam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_cer(actual, hyp):\n",
    "    \"\"\"\n",
    "    Calculate CER using `python-Levenshtein`.\n",
    "    \"\"\"\n",
    "    import Levenshtein as Lev\n",
    "\n",
    "    actual = actual.replace(' ', '')\n",
    "    hyp = hyp.replace(' ', '')\n",
    "    return Lev.distance(actual, hyp) / len(actual)\n",
    "\n",
    "\n",
    "def calculate_wer(actual, hyp):\n",
    "    \"\"\"\n",
    "    Calculate WER using `python-Levenshtein`.\n",
    "    \"\"\"\n",
    "    import Levenshtein as Lev\n",
    "\n",
    "    b = set(actual.split() + hyp.split())\n",
    "    word2char = dict(zip(b, range(len(b))))\n",
    "\n",
    "    w1 = [chr(word2char[w]) for w in actual.split()]\n",
    "    w2 = [chr(word2char[w]) for w in hyp.split()]\n",
    "\n",
    "    return Lev.distance(''.join(w1), ''.join(w2)) / len(actual.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████████████████| 1952/1952 [00:03<00:00, 595.79it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.007769342911655472, 0.0430327868852459)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cers, wers = [], []\n",
    "\n",
    "batch_size = 1\n",
    "pbar = tqdm.tqdm(\n",
    "    range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = encode(test_X[i: index])\n",
    "    batch_y = encode(test_Y[i: index])\n",
    "    feed = {model.X: batch_x,\n",
    "            model.Y: batch_y,}\n",
    "    greedy = sess.run(model.greedy, feed_dict = feed)\n",
    "    predicted = bpe.decode(greedy.tolist())[0].replace('<EOS>', '')\n",
    "    actual = bpe.decode(batch_y)[0].replace('<EOS>', '')\n",
    "    cer = calculate_cer(actual, predicted)\n",
    "    wer = calculate_wer(actual, predicted)\n",
    "    \n",
    "    cers.append(cer)\n",
    "    wers.append(wer)\n",
    "    \n",
    "np.mean(cers), np.mean(wers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'lstm-8/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from lstm-8/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'lstm-8/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/3818740975.py:4: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Variable',\n",
       " 'Variable_1',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'rnn/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/memory_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_0/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/kernel',\n",
       " 'decode/decoder/attention_wrapper/multi_rnn_cell/cell_1/lstm_cell/bias',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel',\n",
       " 'decode/decoder/attention_wrapper/bahdanau_attention/attention_v',\n",
       " 'decode/decoder/attention_wrapper/attention_layer/kernel',\n",
       " 'decode/decoder/dense/kernel',\n",
       " 'decode/decoder/dense/bias',\n",
       " 'decode_1/greedy',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/beam_width',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/start',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range/delta',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/range',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/mul',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims/dim',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/ExpandDims',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/add',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1/shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Reshape_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2/axis',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/GatherV2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/Shape',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_1',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice/stack_2',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/strided_slice',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_probs/output',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids/y',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_word_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_parent_ids',\n",
       " 'decode_2/decoder/while/BeamSearchDecoderStep/next_beam_finished',\n",
       " 'decode_2/beam']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'alphas' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'OptimizeLoss' not in n.name\n",
    "        and 'Global_Step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1070649395.py:3: The name tf.gfile.Exists is deprecated. Please use tf.io.gfile.exists instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1070649395.py:14: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1070649395.py:15: The name tf.train.import_meta_graph is deprecated. Please use tf.compat.v1.train.import_meta_graph instead.\n",
      "\n",
      "INFO:tensorflow:Restoring parameters from lstm-8/model.ckpt\n",
      "INFO:tensorflow:Froze 16 variables.\n",
      "INFO:tensorflow:Converted 16 variables to const ops.\n",
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/1070649395.py:22: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "1649 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph(\"lstm-8\", strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tmp/ipykernel_3105005/3576390908.py:3: The name tf.GraphDef is deprecated. Please use tf.compat.v1.GraphDef instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "g = load_graph('lstm-8/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "i_greedy = g.get_tensor_by_name('import/decode_1/greedy:0')\n",
    "i_beam = g.get_tensor_by_name('import/decode_2/beam:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/tf-nvidia/lib/python3.8/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "test_sess = tf.InteractiveSession(graph=g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "from tensorflow.contrib.seq2seq.python.ops import beam_search_ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-09-17 18:43:20.522645: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying add_default_attributes\n",
      "2022-09-17 18:43:20.529094: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying remove_nodes\n",
      "2022-09-17 18:43:20.534943: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-09-17 18:43:20.535663: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-09-17 18:43:20.543275: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-09-17 18:43:20.543789: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-09-17 18:43:20.549963: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_1/greedy\n",
      "2022-09-17 18:43:20.550526: I tensorflow/tools/graph_transforms/remove_nodes.cc:78] Skipping replacement for decode_2/beam\n",
      "2022-09-17 18:43:20.569831: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_batch_norms\n",
      "2022-09-17 18:43:20.578693: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying fold_old_batch_norms\n",
      "2022-09-17 18:43:20.601538: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying quantize_weights\n",
      "2022-09-17 18:43:20.640561: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying strip_unused_nodes\n",
      "2022-09-17 18:43:20.646244: I tensorflow/tools/graph_transforms/transform_graph.cc:318] Applying sort_by_execution_order\n"
     ]
    }
   ],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']\n",
    "\n",
    "pb = 'lstm-8/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       ['Placeholder'],\n",
    "                                       ['decode_1/greedy', 'decode_2/beam'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/tf-nvidia/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from malaya_boilerplate.huggingface import upload_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lstm-8/\r\n",
      "lstm-8/frozen_model.pb.quantized\r\n",
      "lstm-8/frozen_model.pb\r\n",
      "lstm-8/model.ckpt.meta\r\n",
      "lstm-8/checkpoint\r\n",
      "lstm-8/model.ckpt.index\r\n",
      "lstm-8/model.ckpt.data-00000-of-00001\r\n"
     ]
    }
   ],
   "source": [
    "!tar -cvf output-syllable-base.tar lstm-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/tf-nvidia/lib/python3.8/site-packages/huggingface_hub/hf_api.py:91: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.10. Pass `repo_id` instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "files_mapping = {'output-syllable-base.tar': 'output-syllable-base.tar'}\n",
    "upload_dict(model = 'pretrained-syllable', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mapping = {'lstm-8/frozen_model.pb': 'model.pb'}\n",
    "upload_dict(model = 'syllable-lstm-bahdanau-base', files_mapping = files_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_mapping = {'lstm-8/frozen_model.pb.quantized': 'model.pb'}\n",
    "upload_dict(model = 'syllable-lstm-bahdanau-base-quantized', files_mapping = files_mapping)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1",
   "language": "python",
   "name": "tf1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
